{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a57aefd5",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from string import whitespace, punctuation\n",
    "\n",
    "import numpy as np\n",
    "import pandas\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from zhon.hanzi import punctuation as zh_punctuation\n",
    "import re\n",
    "import jieba\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9b9091c0",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "df = pandas.read_json(\"1.json\")\n",
    "df2 = pandas.DataFrame()\n",
    "business_type = set(df[\"business_type\"])\n",
    "for t in business_type:\n",
    "    df2[t] = df[\"business_type\"].apply(lambda v: int(v == t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4e96cbfe",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#1.去除指定无用的符号\n",
    "df2[\"words\"] = df[\"remarks\"].apply(\n",
    "    lambda v: \"\".join([w for w in v if w not in whitespace + punctuation + zh_punctuation]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da54d972",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#2不能只保留中文，但是要去除无效英文,但是某些英文是有意义的，不能全部去除。->如果是英文字符并且不属于指定要保留的单词的列表，则去除。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "73f96f03",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "##特定字符泛化\n",
    "def clean_number(text):\n",
    "    p=re.sub('\\d+','num',text)\n",
    "    return p\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6e07340c",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#3. 对文本进行jieba分词\n",
    "#如何设置使得该分的分开，不该分的合并？\n",
    "def chinese_tok(text):\n",
    "#     s=clean_number(text)\n",
    "    jieba.suggest_freq('副卡',True)\n",
    "    jieba.suggest_freq('补卡',True)\n",
    "    jieba.suggest_freq('主卡',True)\n",
    "    jieba.suggest_freq('G',True)\n",
    "    jieba.suggest_freq('元',True)\n",
    "    jieba.suggest_freq('NUM',True)\n",
    "    dc=[t for t in jieba.cut(text)]\n",
    "    return \" \".join(dc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "da1152de",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['杜丽颖15129742266过户杜选卫13991035655名下15129742266当副卡',\n",
       " '用户申请将17762133367单卡融合到5G239融合套餐中受理人联系电话13389109931',\n",
       " '补卡18191011778卡号8986032124910547094']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.sample(3)['words'].values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "9725507e",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "f=df2.sample(3)['words'].values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "bb26e02e",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "号码num迁转num元大流量存num赠num元包numGnum元包numG客户经理唐振num\n",
      "加装num元云电脑一台邮箱号num\n",
      "num天翼AI连锁版经济型月付套餐num邮箱aycnumcom地址XJJSXZYYGFnum\n"
     ]
    }
   ],
   "source": [
    "for i in f:\n",
    "    print(clean_number(i))\n",
    "    \n",
    "f=[clean_number(i) for i in f]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "379da799",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "号码 num 迁转 num 元大 流量 存 num 赠 num 元包 numGnum 元包 numG 客户经理 唐振 num\n",
      "加装 num 元云 电脑 一台 邮箱 号 num\n",
      "num 天翼 AI 连锁 版 经济型 月付 套餐 num 邮箱 aycnumcom 地址 XJJSXZYYGFnum\n"
     ]
    }
   ],
   "source": [
    "for i in f:\n",
    "    print(chinese_tok(i))\n",
    "    \n",
    "f=[chinese_tok(i) for i in f]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "fd93fab6",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 分词后再处理停用词.\n",
    "# 什么词是停用词？\n",
    "# 4,停用词表\n",
    "stopwords = ['的','呀','这','那','就','的话','如果','元','包']\n",
    "def drop_stopwords(words_cut_list, stopwords):\n",
    "    contents_clean = []\n",
    "    for line in words_cut_list:\n",
    "        line_clean = []\n",
    "        for word in line:\n",
    "            if word in stopwords:\n",
    "                continue\n",
    "            line_clean.append(word)\n",
    "        contents_clean.append(\"\".join(line_clean))\n",
    "    return contents_clean\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "5572ec34",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['号码', 'num', '迁转', 'num', '元大', '流量', '存', 'num', '赠', 'num', '元包', 'numGnum', '元包', 'numG', '客户经理', '唐振', 'num']\n",
      "['号码', 'num', '迁转', 'num', '大', '流量', '存', 'num', '赠', 'num', '包', 'numGnum', '包', 'numG', '客户经理', '唐振', 'num']\n",
      "['加装', 'num', '元云', '电脑', '一台', '邮箱', '号', 'num']\n",
      "['加装', 'num', '云', '电脑', '一台', '邮箱', '号', 'num']\n",
      "['num', '天翼', 'AI', '连锁', '版', '经济型', '月付', '套餐', 'num', '邮箱', 'aycnumcom', '地址', 'XJJSXZYYGFnum']\n",
      "['num', '天翼', 'AI', '连锁', '版', '经济型', '月付', '套餐', 'num', '邮箱', 'aycnumcom', '地址', 'XJJSXZYYGFnum']\n"
     ]
    }
   ],
   "source": [
    "for i in f:\n",
    "    print(i.split(\" \"))\n",
    "    res=drop_stopwords(i.split(\" \"),stopwords)\n",
    "    print(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac1b1557",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}